e026674a16aafeedbdd9112a30402f20ba2c4e50,src/edu/stanford/nlp/pipeline/ChineseSegmenterAnnotator.java,ChineseSegmenterAnnotator,splitCharacters,#CoreMap#,129
Before Change
// if this word is a whitespace or a control character, set 'seg' to true for next word, and break
if (origText.charAt(i) != '\n' && (Character.isSpaceChar(origText.charAt(i)) || Character.isISOControl(origText.charAt(i))
|| Character.isWhitespace(origText.charAt(i)))) {
seg = true;
} else {
// if this word is a word, put it as a feature label and set seg to false for next word
After Change
// if this word is a whitespace or a control character, set 'seg' to true for next word, and break
if ((Character.isSpaceChar(origText.charAt(i)) || Character.isISOControl(origText.charAt(i))) &&
! (origText.charAt(i) == '\n' || origText.charAt(i) == '\r')) {
seg = true;
} else if (Character.isISOControl(origText.charAt(i))) {
// skip it but don't set seg
seg = false;
} else {